package com.li.spark.optimization

import org.apache.spark.{SparkConf, SparkContext}

object ForeachPartitionOp {
  def main(args: Array[String]): Unit = {
    val sc: SparkContext = getSparkContext

    val dataRDD = sc.parallelize(Array(1, 2, 3, 4, 5))
    //foreachPartition 一次处理一个分区的数据
    val sumPartiotions = dataRDD.foreachPartition(it => {
      it.foreach(item => {
        println(item)
      })
    })

    sc.stop();
  }


  private def getSparkContext = {
    //创建SparkContext
    val conf = new SparkConf();
    conf.setAppName("ForeachPartitionOp").setMaster("local");
    new SparkContext(conf);
  }
}
